#!/usr/bin/env python  
# -*- coding: utf-8 -*-  
from HTMLParser import HTMLParser
import sys,re,urllib2,urllib,time,os,json,codecs,csv

type = sys.getfilesystemencoding() 
reload(sys)  
sys.setdefaultencoding('utf8')

class Yohoo:
    #开始方法
    def __init__(self):
        self.page = 1
        self.date_items = []
        self.date_list_items = []
        self.star_time = 16
        self.end_time = 31
        self.overflow_exist = 1
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers    = { 'User-Agent' : self.user_agent }
        self.tab = ''

    def start(self,tab):
        if(self.tab != tab ):
            self.overflow_exist = 1
            self.page = 1
            self.date_items = []
            self.date_list_items = []

        self.tab = tab
        # 获取用户数据
        url        = 'https://autos.yahoo.com.tw/ajax/api_car_news?page=%s'%self.page+'&news-tab=%s'%self.tab+''
        request    = urllib2.Request(url, headers = self.headers)
        response   = urllib2.urlopen(request).read()
        msg        = json.loads(response)

        self.fnFiltrateData(msg)

    def fnFiltrateData(self,msg):
        get_date      = re.findall('<br>(.*?)</span>', msg['html'])
        get_url     = re.findall('<a.*?href?\s*=\s*\"+?(.*?)\"', msg['html'])
        
        for i in range(len(get_date)):
            if(self.end_time >= int(get_date[i]) >= self.star_time ):
                self.overflow_exist = 2
                self.date_items.append(['%s'%get_date[i],'%s'%get_url[i]])
            elif self.overflow_exist == 2:
                print "----------------------------------"
                self.overflow_exist = 3
                break


        if(self.overflow_exist != 3 and self.overflow_exist != 4):
            print u"正在捉取第 %s 页鏈接"%self.page
            self.page += 1
            print u"當前的頻道%s"%self.tab
            self.start(self.tab)
        elif self.overflow_exist != 4:
            print len(self.date_items)
            print u"一共%s條數據"%len(self.date_items)
            if (self.date_items):
                for item in self.date_items:
                    self.fnGetDetailData(item[1])
                self.fnCreateCsv()

    def fnGetDetailData(self,url):
        
        request    = urllib2.Request(url, headers = self.headers)
        response   = urllib2.urlopen(request).read()
        page_html =  response
        month_day = time.strftime("%y-%m-%d")
        title = re.findall('<h1 class="title">(.*?)</h1>',page_html)
        source = re.findall('<div class="detail">\n.*?<span>(.*?)</span>',page_html)
        dateOrauthor = re.findall('<div class="detail">\n.*?<span>.*?</span>\n.*?<span>(.*?)</span>',page_html)
        date = dateOrauthor[0][len(dateOrauthor[0])-19:len(dateOrauthor[0])]
        author = dateOrauthor[0][0:len(dateOrauthor[0])-20]
        
        print u"開始爬取文章詳情頁-%s"%source[0]
        print month_day[0:5]
        print date[2:7]
        if("18-05" == date[2:7]):
            items = [author,date,title[0],source[0],url]
            self.date_list_items.append(items)

    

    def fnCreateCsv(self):
        if (self.date_list_items):
            print u"Yohoo-%s-%s-%s號 共%s條內容 生成excel"%(self.tab,self.star_time,self.end_time,len(self.date_items))
            # 写入数据
            for item in self.date_list_items:
                if os.path.exists(u'./Yohoo-%s-%s-%s號共%s篇.csv'%(self.tab,self.star_time,self.end_time,len(self.date_items))) == False:
                    with codecs.open(u'./Yohoo-%s-%s-%s號共%s篇.csv'%(self.tab,self.star_time,self.end_time,len(self.date_items)), 'w', 'utf_8_sig') as f:
                        f.write('作者,时间,标题,來源,鏈接')
                with codecs.open(u'./Yohoo-%s-%s-%s號共%s篇.csv'%(self.tab,self.star_time,self.end_time,len(self.date_items)), 'a', 'utf_8') as f:
                    f.write(u'\r\n%s,%s,%s,%s,%s'%(item[0],item[1],item[2],item[3],item[4]))
        else:
            print u'未找到数据'


spider = Yohoo()
channel_array = [
    '汽車新聞', 
    '新車試駕',
    '特別企劃',
    '賽車報導',
]

for i in channel_array:
    spider.start(i)

